--- title: Atmospheric Correction keywords: fastai sidebar: home_sidebar summary: "Out in the field, the OpenHSI camera measures light as it is reflected off surfaces. This light from the sun has absorption features from molecules/aerosols in the atmosphere and scattering. After converting the raw digital number counts into radiance" description: "Out in the field, the OpenHSI camera measures light as it is reflected off surfaces. This light from the sun has absorption features from molecules/aerosols in the atmosphere and scattering. After converting the raw digital number counts into radiance" nb_path: "03_atmos.ipynb" ---
Find the closest station at http://weather.uwyo.edu/upperair/sounding.html
to use the atmospheric sounding on the day.
find the station number and region code (for example, south pacific is pac and new zealand is nz)
Default is Willis Island in Queensland, Australia.
https://py6s.readthedocs.io/en/latest/helpers.html#importing-atmospheric-profiles-from-radiosonde-data
{% include warning.html content='Py6S states that custom aerosol profiles can be input but this is broken and there is no easy fix. Don’t waste a day trying... ' %}
Py6S provides a method to loop over an array and return the 6SV result. However, it does not have a progress bar. For something that can take several minutes, not knowing how long it will take is genuinely frustrating. Therefore, we provide a modified version of SixSHelpers.Wavelengths.run_wavelengths called Model6SV.run_wavelengths that has a progress bar.
To use 6SV, Py6S will write input files and parse the output files. All this I/O is slow but we got to deal with it. Py6S is clever about it though and uses threading to do some compute while waiting for files. The modified version follows the same method except with the addition of a callback to update the progress bar when interations are done.
model = Model6SV(wavelength_array=np.arange(350, 1000, 10))
model.show()
dc = DataCube()
dc.load_nc("../load_multiple/2021-05-26 03_26_26.011211 dn.nc")
dc.show("bokeh",robust=True)
model = Model6SV(wavelength_array=dc.binned_wavelengths)
model.show()
spec_matcher = SpectralMatcher(speclib_path="assets/speclib.pkl",model_6SV=model)
%time spec_matcher.topk_spectra(dc.dc.data[450,1500,:],5,refine=True)
spec_matcher.show()
elc = ELC(nc_path="../load_multiple/2021-05-26 03_26_26.011211 radiance float32.nc",speclib_path="assets/speclib.pkl",model_6SV=model)
elc()
plt.subplot(121); plt.plot(elc.a_ELC)
plt.subplot(122); plt.plot(elc.b_ELC)
df = pd.read_pickle("assets/openhsi_radiance_select_targets.pkl")
df.plot(x='wavelength (nm)', y=df.columns[1:],figsize=(8,6),
xlabel="wavelength (nm)",ylabel="radiance (uW/cm^2/sr/nm)",
ylim=(0,None)).legend(loc="upper right",fontsize='xx-small')
speclib2 = elc.speclib.copy()
speclib2.rename(columns={"gray_small":"gray small","green_small":"green","cyan_small":"cyan","red_small":"magenta"},inplace=True)
name_dict = {"gray uni":"gray small","cyan uni":"cyan","green uni":"green",
"magenta uni":"magenta","charcoal":"charcoal","gray":"gray","blue":"blue",
"blue takeoff":"blue_take_off"}
name_dict2 = dict((v,k) for k,v in name_dict.items())
from datetime import datetime
model2 = Model6SV(wavelength_array=dc.binned_wavelengths,z_time=datetime.strptime("2021-05-26 03:26","%Y-%m-%d %H:%M"))
model2.show()
def get_3curves(df_col_name="blue"):
#sim_df2 = spec_matcher_rad.topk_spectra(df[df_col_name],1)
# LAB
plot0 = hv.Curve(zip(df["wavelength (nm)"],speclib2[df_col_name]),label="Lab ASD").opts(
fontsize={'title': 15, 'labels': 14, 'xticks': 10, 'yticks': 10,})
# ELC
spect = (np.array(df[name_dict2[df_col_name]]) - elc.b_ELC)/elc.a_ELC
plot1 = hv.Curve(zip(df["wavelength (nm)"],spect),label="ELC estimate").opts(line_dash='dashed')
# 6SV
spect2 = np.array(df[name_dict2[df_col_name]])/model2.radiance*10
plot2 = hv.Curve(zip(df["wavelength (nm)"],spect2),label="6SV estimate").opts(line_dash='dotted')
# Modtran
#plot3 = hv.Curve(zip(modtran_df["wavelength (nm)"],modtran_df[df_col_name]),label="Modtran")
return (plot0 * plot1 * plot2 ).opts(xlabel="wavelength (nm)",ylabel="reflectance").opts(
title=df_col_name,ylim=(0,1),legend_position='top_left').opts({'Curve': {'color': hv.Cycle('Dark2')}})
( get_3curves("gray") + get_3curves("charcoal") + get_3curves("blue") + \
get_3curves("green") + get_3curves("magenta") + get_3curves("cyan") ).cols(3)
Compare against USGS spectral library
from pathlib import Path
@patch
def show(self:SpectralMatcher,is_ref:bool=False,ref_est:bool=False):
if self.topk_idx is not None:
hv_curves = []
if not is_ref:
hv_curves.append( hv.Curve(zip(self.wavelengths,self.last_spectra),label="tap point") )
elif ref_est:
hv_curves.append( hv.Curve(zip(self.wavelengths,self.last_spectra/(self.model_6SV.radiance/10)),label="6SV estimate") )
for l in self.sim_df["label"]:
alpha = self.sim_df[self.sim_df["label"]==l]["score"].to_numpy()
alpha = 0.2 if len(alpha) == 0 else alpha[0]
thresh = 0.94 if self.refine else 0.99
alpha = 0.2 if alpha < thresh else 0.9
temp = self.speclib_ref.copy() if is_ref else self.spectra.copy()
temp.insert(0,"wavelength",self.wavelengths)
hv_curves.append( hv.Curve(temp,kdims="wavelength",vdims=l,label=l).opts(alpha=alpha) )
return hv.Overlay(hv_curves).opts(width=1000, height=600,xlabel="wavelength (nm)",ylabel="reflectance")
else:
temp = self.speclib_ref.copy() if is_ref else self.spectra.copy()
temp.insert(0,"wavelength",self.wavelengths)
curve_list = [hv.Curve(temp,kdims="wavelength",vdims=i,label=i) for i in temp.columns[1:] if (np.sum(temp[i]>1.)>0)]
print(curve_list)
if getattr(self,"last_spectra",None):
return (hv.Overlay(curve_list)*hv.Curve(zip(self.wavelengths,self.last_spectra),label="last spectra")).opts(
width=1000, height=600,xlabel="wavelength (nm)",ylabel="reflectance",ylim=(0,1.1))
return (hv.Overlay(curve_list)).opts(width=1000, height=600,xlabel="wavelength (nm)",ylabel="reflectance",ylim=(0,1.1))
@patch
def update_lib_folder(self:SpectralMatcher,directory,sort=False,save=False,show=True):
fnames = sorted(os.listdir(directory))
cwd = Path(directory)
temp = None
for f in fnames:
if "ASD" not in f: continue
if temp is None:
temp = pd.read_csv(cwd/f,delimiter="\t")
temp.rename({temp.columns[0]:f[9:-9]}, axis='columns',inplace=True)
temp.loc[~(temp[temp.columns[0]] > 0), temp.columns[0]]=np.nan
if (np.sum(temp<0.)>0).to_numpy()[0]:
breakpoint()
#col_name = temp.columns[0]
#temp.assign(col_name = lambda x: x.col_name.where(x.col_name.ge(0)))
else:
buff = pd.read_csv(cwd/f,delimiter="\t")
buff.loc[~(buff[buff.columns[0]] > 0), buff.columns[0]]=np.nan
temp.insert(len(temp.columns),f[9:-9],buff[buff.columns[0]])
self.orig_speclib = pd.concat([self.orig_speclib,temp[temp.columns[1:]]],axis=1)
self.orig_speclib = self.orig_speclib.loc[:,~self.orig_speclib.columns.duplicated()]
print(f"Added folder of ASD spectra to spectral library")
self._interp()
return self._sort_save_show(sort,save,show)
%%time
spec_matcher = SpectralMatcher(speclib_path="assets/speclib.pkl",model_6SV=model)
spec_matcher.update_lib_folder("../../Downloads/ASCIIdata_splib07b/ChapterV_Vegetation/")
spec_matcher.show(is_ref=True).opts(tools=["hover"])
spec_matcher.orig_speclib["Cactus_Opuntia-1_purple_pad_ASDFRa"].plot()
spec_matcher.orig_speclib["Walnut_Leaf_SUN_(Green)_BECKa"]
spec_matcher.orig_speclib[["wavelength",'Walnut_Leaf_SUN_(Green)_BECKa']].plot(x="wavelength",xlim=(350,2500))
@patch
def _interp(self):
# interpolate the spectral library to the OpenHSI wavelengths
self.speclib = self.orig_speclib.copy()
self.speclib.insert(0,"type","USGS")
self.speclib = pd.concat( [ pd.DataFrame({"type":"openhsi","wavelength":self.wavelengths}), self.speclib] )
self.speclib.set_index("wavelength",inplace=True)
self.speclib.interpolate(method="cubicspline",axis="index",limit_direction="both",inplace=True)
self.speclib = self.speclib[self.speclib["type"].str.match("openhsi")]
self.speclib.drop("type", 1,inplace=True)
self.speclib_ref = self.speclib.copy()
self.spectra = (self.speclib[:].T*self.model_6SV.radiance/10).T
self.spectra_norm = norm(self.spectra,axis=0)
speclib = spec_matcher.orig_speclib[["wavelength","Walnut_Leaf_SUN_(Green)_BECKa"]]
speclib.insert(0,"type","USGS") # insert column in 0th index with label "type" and value "USGS"
speclib = pd.concat( [ pd.DataFrame({"type":"openhsi","wavelength":spec_matcher.wavelengths}), speclib] )
speclib.set_index("wavelength",inplace=True)
speclib.tail(2000)
print(spec_matcher.orig_speclib.columns.to_numpy())